#coding:utf-8
import os.path,codecs,re,types
import hashlib
from pyquery import PyQuery as pq
class FileUtil(object):
    #dirname 必须跟当前这个文件 的目录同级
    def __init__(self,dirname):
        self.spiderdir=dirname
        self.dirpath= os.path.join(os.path.dirname(os.getcwd()),self.spiderdir)
        os.chdir(self.dirpath)#切换工作目录
    #写文件
    def write_to_file(self,filename,content,encoding="utf-8"):
        with codecs.open(self.join_path(filename),"w", encoding) as f:
         
            return f.write(content)
    #删除文件
    def del_file(self,filename):
        return os.remove(self.join_path(filename))
    #获取md5值
    def get_md5_filename(self,filename):
        content=self.read_from_file(filename)
        return self.get_md5_content(content)
    #根据内容获取md5值
    def get_md5_content(self,content):
        m2 = hashlib.md5()   
        m2.update(content)   
        return m2.hexdigest()
    #文件内容跟数据库中的内容MD5值比较
    def compare_md5_content(self,filename,content):
        if self.file_exists(filename):
            file_md5=self.get_md5_filename(filename)
            db_content=replaceCharEntity(content)
            db_md5=self.get_md5_content(db_content)
            
            return db_md5==file_md5
        else:
            return False
        
    #读文件
    def read_from_file(self,filename,encoding="utf-8"):
        with codecs.open(self.join_path(filename),"r", encoding) as f:
            return f.read()
    #判定文件是否存在
    def file_exists(self,filename):
        return os.path.exists(self.join_path(filename))
    #获取文件扩展名
    def get_ext_name(self,filename):
        return os.path.splitext(self.join_path(filename))[-1]
    #获取文件名
    def get_file_name(self,filename):
        return os.path.splitext(filename)[0]
    #列出目录下，指定扩展名 到所有文件
    def list_file(self,ext=".py"):
        file_list=os.listdir(self.dirpath)
        newlist = []
        for file_s in file_list:
            if file_s.endswith(ext) and not self.get_file_name(file_s)=="__init__":
                newlist.append(file_s)
        return newlist
    #根据爬虫文件，提取爬虫的名字
    def get_spider_name(self,filename):
        content=self.read_from_file(filename)
        re_result=re.search(r'name(\s+)?=(\s+)?"\w+"', content)
        if re_result is not None:
            return re.search("\w+",re.search('(\s+)?"\w+"',re_result.group(0)).group(0).strip()).group(0).strip()
        else:
            return None
    def is_file(self,filename):
        return os.path.isfile(self.join_path(filename))
    #合并路径
    def join_path(self,filename):
        return os.path.join(self.dirpath,filename)

#html parse 去掉html转义符 
def replaceCharEntity(content):
        CHAR_ENTITIES={'nbsp':' ','160':' ',
                    'lt':'<','60':'<',
                    'gt':'>','62':'>',
                    'amp':'&','38':'&',
                    'quot':'"','34':'"',}
         
        re_charEntity=re.compile(r'&#?(?P<name>\w+);')
        sz=re_charEntity.search(content)
        while sz:
            #entity=sz.group()#entity全称，如>
            key=sz.group('name')#去除&;后entity,如>为gt
            try:
                content=re_charEntity.sub(CHAR_ENTITIES[key],content,1)
                sz=re_charEntity.search(content)
            except KeyError:
                #以空串代替
                content=re_charEntity.sub('',content,1)
                sz=re_charEntity.search(content)
        return content

def is_None(collection):
    if type(collection)==types.ListType or type(collection)==types.DictType:
        return None if len(collection)<=0 else not None
    else:
        raise ValueError("参数类型错误")

def get_text(htmlstr,extra=None):
    if extra is not None:
        return pq(htmlstr)(extra).text()
    else:
        return pq(htmlstr).text()
    
if __name__=="__main__":
    
    fileutil=FileUtil("spiders_template")
    
    
    print fileutil.get_md5_filename("baidu_spider.py")
    print fileutil.get_spider_name("sjshdy_spider.py")
    print fileutil.get_ext_name("baidu_spider.py")
    print fileutil.get_file_name("baidu_spider.py")
    print replaceCharEntity("https://www.google.com.hk/search?q=%E5%A4%A7%E6%95%B0%E6%8D%AE&amp;safe=strict&amp;hl=zh-CN&amp;gl=cn&amp;tbm=nws&amp;start=0&amp;dpr=1")
    print fileutil.list_file()
#     print fileutil.write_to_file("xiong.py", "sssss")
#     print fileutil.file_exists("xiong.py")
    #print fileutil.del_file("xiong.py")
#     print os.getcwd()
#     print os.curdir
#     print os.pardir
#     print os.path.abspath(os.curdir)
# 
#     print os.path.abspath('.')
#     print __file__
#     print os.path.dirname(os.getcwd())
#     path= os.path.join(os.path.dirname(os.getcwd()),"spiders")
#     print path
#     os.chdir(path)
#     print os.getcwd()
#     print os.path.dirname(os.getcwd())
    